{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## logistic regression multi-dimensional data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " logistic regression multi-dimensional data\n",
    " \n",
    " \n",
    " $$ F(X)=X \\times W $$\n",
    " $$ H(x)= \\frac{1}{1+ e ^{-F(x)}} $$\n",
    " $$ C= -\\frac{1}{n} \\sum_{i,j} (Y \\odot log(H(x)) + (1-Y) \\odot log(1-H(x)) ) $$\n",
    "\n",
    "$X_{n \\times k}$\n",
    "\n",
    "$W_{k \\times p}$\n",
    "\n",
    "$Y_{n \\times p}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "n, k, p=100, 8, 3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=np.random.random([n,k])\n",
    "W=np.random.random([k,p])\n",
    "\n",
    "y=np.random.randint(p, size=(1,n))\n",
    "Y=np.zeros((n,p))\n",
    "Y[np.arange(n), y]=1\n",
    "\n",
    "max_itr=5000\n",
    "alpha=0.01\n",
    "Lambda=0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gradient is as follows:\n",
    "$$ X^T (H(x)-Y) + \\lambda 2 W$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# F(x)= w[0]*x + w[1]\n",
    "def F(X, W):\n",
    "    return np.matmul(X,W)\n",
    "\n",
    "def H(F):\n",
    "    return 1/(1+np.exp(-F))\n",
    "\n",
    "def cost(Y_est, Y):\n",
    "    E= - (1/n) * (np.sum(Y*np.log(Y_est) + (1-Y)*np.log(1-Y_est)))  + np.linalg.norm(W,2)\n",
    "    return E, np.sum(np.argmax(Y_est,1)==y)/n\n",
    "\n",
    "def gradient(Y_est, Y, X):\n",
    "    return (1/n) * np.matmul(X.T, (Y_est - Y) ) + Lambda* 2* W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(W, X, Y, alpha, max_itr):\n",
    "    for i in range(max_itr):\n",
    "        \n",
    "        F_x=F(X,W)\n",
    "        Y_est=H(F_x)\n",
    "        E, c= cost(Y_est, Y)\n",
    "        Wg=gradient(Y_est, Y, X)\n",
    "        W=W - alpha * Wg\n",
    "        if i%1000==0:\n",
    "            print(E, c)\n",
    "        \n",
    "    return W, Y_est"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To take into account for the biases, we concatenate X by a 1 column, and increase the number of rows in W by one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.368653735228364 0.31\n",
      "4.994251188297815 0.43\n",
      "4.951873226767272 0.48\n",
      "4.922370610237865 0.47\n",
      "4.901694423284286 0.48\n"
     ]
    }
   ],
   "source": [
    "X=np.concatenate( (X, np.ones((n,1))), axis=1 ) \n",
    "W=np.concatenate( (W, np.random.random((1,p)) ), axis=0 )\n",
    "\n",
    "W, Y_est = fit(W, X, Y, alpha, max_itr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
